{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use OffPAC to Play Acrobot-v1\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import imp\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras\n",
    "from tensorflow import nn\n",
    "from tensorflow import optimizers\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import losses\n",
    "\n",
    "imp.reload(logging)\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19:57:00 [INFO] env: <AcrobotEnv<Acrobot-v1>>\n",
      "19:57:00 [INFO] action_space: Discrete(3)\n",
      "19:57:00 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)\n",
      "19:57:00 [INFO] reward_range: (-inf, inf)\n",
      "19:57:00 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}\n",
      "19:57:00 [INFO] _max_episode_steps: 500\n",
      "19:57:00 [INFO] _elapsed_steps: None\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Acrobot-v1')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OffPACAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.actor_net = self.build_net(hidden_sizes=[100,],\n",
    "                output_size=self.action_n,\n",
    "                output_activation=nn.softmax, learning_rate=0.0002)\n",
    "        self.critic_net = self.build_net(hidden_sizes=[100,],\n",
    "                output_size=self.action_n,\n",
    "                learning_rate=0.0004)\n",
    "\n",
    "    def build_net(self, hidden_sizes, output_size,\n",
    "                activation=nn.relu, output_activation=None,\n",
    "                loss=losses.mse, learning_rate=0.001):\n",
    "        model = keras.Sequential()\n",
    "        for hidden_size in hidden_sizes:\n",
    "            model.add(layers.Dense(units=hidden_size,\n",
    "                    activation=activation))\n",
    "        model.add(layers.Dense(units=output_size,\n",
    "                activation=output_activation))\n",
    "        optimizer = optimizers.SGD(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.discount = 1.\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if self.mode == 'train':\n",
    "            action = np.random.choice(self.action_n)\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                self.learn()\n",
    "            self.discount *= self.gamma\n",
    "        else:\n",
    "            probs = self.actor_net.predict(observation[np.newaxis])[0]\n",
    "            action = np.random.choice(self.action_n, p=probs)\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def learn(self):\n",
    "        state, _, _, action, next_state, reward, done, next_action = \\\n",
    "                self.trajectory[-8:]\n",
    "        behavior_prob = 1. / self.action_n\n",
    "        pi = self.actor_net.predict(state[np.newaxis])[0, action]\n",
    "        ratio = pi / behavior_prob # importance sampling ratio\n",
    "\n",
    "        # train actor\n",
    "        q = self.critic_net.predict(state[np.newaxis])[0, action]\n",
    "        state_tensor = tf.convert_to_tensor(state[np.newaxis], dtype=tf.float32)\n",
    "        with tf.GradientTape() as tape:\n",
    "            pi_tensor = self.actor_net(state_tensor)[0, action]\n",
    "            actor_loss_tensor = -self.discount * q / behavior_prob * pi_tensor\n",
    "        grad_tensors = tape.gradient(actor_loss_tensor, self.actor_net.variables)\n",
    "        self.actor_net.optimizer.apply_gradients(zip(grad_tensors,\n",
    "                self.actor_net.variables))\n",
    "\n",
    "        # train critic\n",
    "        next_q = self.critic_net.predict(next_state[np.newaxis])[0, next_action]\n",
    "        target = reward + (1. - done) * self.gamma * next_q\n",
    "        target_tensor = tf.convert_to_tensor(target, dtype=tf.float32)\n",
    "        with tf.GradientTape() as tape:\n",
    "            q_tensor = self.critic_net(state_tensor)\n",
    "            mse_tensor = losses.MSE(target_tensor, q_tensor)\n",
    "            critic_loss_tensor = ratio * mse_tensor\n",
    "        grad_tensors = tape.gradient(critic_loss_tensor, self.critic_net.variables)\n",
    "        self.critic_net.optimizer.apply_gradients(zip(grad_tensors,\n",
    "                self.critic_net.variables))\n",
    "\n",
    "\n",
    "agent = OffPACAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19:57:01 [INFO] ==== train ====\n",
      "20:00:48 [DEBUG] train episode 0: reward = -500.00, steps = 500\n",
      "20:04:17 [DEBUG] train episode 1: reward = -321.00, steps = 322\n",
      "20:07:49 [DEBUG] train episode 2: reward = -329.00, steps = 330\n",
      "20:11:13 [DEBUG] train episode 3: reward = -275.00, steps = 276\n",
      "20:14:49 [DEBUG] train episode 4: reward = -256.00, steps = 257\n",
      "20:19:41 [DEBUG] train episode 5: reward = -440.00, steps = 441\n",
      "20:23:54 [DEBUG] train episode 6: reward = -307.00, steps = 308\n",
      "20:28:46 [DEBUG] train episode 7: reward = -267.00, steps = 268\n",
      "20:33:15 [DEBUG] train episode 8: reward = -221.00, steps = 222\n",
      "20:37:46 [DEBUG] train episode 9: reward = -188.00, steps = 189\n",
      "20:42:54 [DEBUG] train episode 10: reward = -500.00, steps = 500\n",
      "20:47:11 [DEBUG] train episode 11: reward = -200.00, steps = 201\n",
      "20:52:08 [DEBUG] train episode 12: reward = -500.00, steps = 500\n",
      "20:57:07 [DEBUG] train episode 13: reward = -500.00, steps = 500\n",
      "21:02:21 [DEBUG] train episode 14: reward = -500.00, steps = 500\n",
      "21:07:57 [DEBUG] train episode 15: reward = -500.00, steps = 500\n",
      "21:12:58 [DEBUG] train episode 16: reward = -500.00, steps = 500\n",
      "21:17:09 [DEBUG] train episode 17: reward = -199.00, steps = 200\n",
      "21:22:13 [DEBUG] train episode 18: reward = -436.00, steps = 437\n",
      "21:26:58 [DEBUG] train episode 19: reward = -223.00, steps = 224\n",
      "21:31:21 [DEBUG] train episode 20: reward = -132.00, steps = 133\n",
      "21:35:45 [DEBUG] train episode 21: reward = -157.00, steps = 158\n",
      "21:39:51 [DEBUG] train episode 22: reward = -158.00, steps = 159\n",
      "21:43:56 [DEBUG] train episode 23: reward = -133.00, steps = 134\n",
      "21:48:06 [DEBUG] train episode 24: reward = -155.00, steps = 156\n",
      "21:52:25 [DEBUG] train episode 25: reward = -160.00, steps = 161\n",
      "21:56:41 [DEBUG] train episode 26: reward = -110.00, steps = 111\n",
      "22:00:51 [DEBUG] train episode 27: reward = -102.00, steps = 103\n",
      "22:04:42 [DEBUG] train episode 28: reward = -121.00, steps = 122\n",
      "22:08:33 [DEBUG] train episode 29: reward = -150.00, steps = 151\n",
      "22:12:26 [DEBUG] train episode 30: reward = -96.00, steps = 97\n",
      "22:16:32 [DEBUG] train episode 31: reward = -115.00, steps = 116\n",
      "22:20:46 [DEBUG] train episode 32: reward = -116.00, steps = 117\n",
      "22:25:12 [DEBUG] train episode 33: reward = -113.00, steps = 114\n",
      "22:29:31 [DEBUG] train episode 34: reward = -122.00, steps = 123\n",
      "22:34:22 [DEBUG] train episode 35: reward = -119.00, steps = 120\n",
      "22:34:22 [INFO] ==== test ====\n",
      "22:34:39 [DEBUG] test episode 0: reward = -123.00, steps = 124\n",
      "22:34:56 [DEBUG] test episode 1: reward = -122.00, steps = 123\n",
      "22:35:08 [DEBUG] test episode 2: reward = -96.00, steps = 97\n",
      "22:35:19 [DEBUG] test episode 3: reward = -87.00, steps = 88\n",
      "22:35:29 [DEBUG] test episode 4: reward = -96.00, steps = 97\n",
      "22:35:41 [DEBUG] test episode 5: reward = -92.00, steps = 93\n",
      "22:35:51 [DEBUG] test episode 6: reward = -80.00, steps = 81\n",
      "22:36:05 [DEBUG] test episode 7: reward = -113.00, steps = 114\n",
      "22:36:17 [DEBUG] test episode 8: reward = -88.00, steps = 89\n",
      "22:36:31 [DEBUG] test episode 9: reward = -114.00, steps = 115\n",
      "22:36:41 [DEBUG] test episode 10: reward = -85.00, steps = 86\n",
      "22:36:54 [DEBUG] test episode 11: reward = -120.00, steps = 121\n",
      "22:37:05 [DEBUG] test episode 12: reward = -92.00, steps = 93\n",
      "22:37:18 [DEBUG] test episode 13: reward = -111.00, steps = 112\n",
      "22:37:33 [DEBUG] test episode 14: reward = -121.00, steps = 122\n",
      "22:38:10 [DEBUG] test episode 15: reward = -258.00, steps = 259\n",
      "22:38:29 [DEBUG] test episode 16: reward = -138.00, steps = 139\n",
      "22:38:41 [DEBUG] test episode 17: reward = -97.00, steps = 98\n",
      "22:38:53 [DEBUG] test episode 18: reward = -96.00, steps = 97\n",
      "22:39:05 [DEBUG] test episode 19: reward = -92.00, steps = 93\n",
      "22:39:19 [DEBUG] test episode 20: reward = -110.00, steps = 111\n",
      "22:39:31 [DEBUG] test episode 21: reward = -102.00, steps = 103\n",
      "22:39:43 [DEBUG] test episode 22: reward = -100.00, steps = 101\n",
      "22:39:53 [DEBUG] test episode 23: reward = -85.00, steps = 86\n",
      "22:40:06 [DEBUG] test episode 24: reward = -104.00, steps = 105\n",
      "22:40:17 [DEBUG] test episode 25: reward = -90.00, steps = 91\n",
      "22:40:29 [DEBUG] test episode 26: reward = -106.00, steps = 107\n",
      "22:40:44 [DEBUG] test episode 27: reward = -119.00, steps = 120\n",
      "22:40:56 [DEBUG] test episode 28: reward = -104.00, steps = 105\n",
      "22:41:09 [DEBUG] test episode 29: reward = -106.00, steps = 107\n",
      "22:41:24 [DEBUG] test episode 30: reward = -128.00, steps = 129\n",
      "22:41:39 [DEBUG] test episode 31: reward = -120.00, steps = 121\n",
      "22:42:02 [DEBUG] test episode 32: reward = -211.00, steps = 212\n",
      "22:42:13 [DEBUG] test episode 33: reward = -102.00, steps = 103\n",
      "22:42:22 [DEBUG] test episode 34: reward = -80.00, steps = 81\n",
      "22:42:34 [DEBUG] test episode 35: reward = -105.00, steps = 106\n",
      "22:42:46 [DEBUG] test episode 36: reward = -108.00, steps = 109\n",
      "22:42:59 [DEBUG] test episode 37: reward = -108.00, steps = 109\n",
      "22:43:13 [DEBUG] test episode 38: reward = -122.00, steps = 123\n",
      "22:43:24 [DEBUG] test episode 39: reward = -101.00, steps = 102\n",
      "22:43:35 [DEBUG] test episode 40: reward = -101.00, steps = 102\n",
      "22:43:46 [DEBUG] test episode 41: reward = -95.00, steps = 96\n",
      "22:43:58 [DEBUG] test episode 42: reward = -112.00, steps = 113\n",
      "22:44:09 [DEBUG] test episode 43: reward = -98.00, steps = 99\n",
      "22:44:20 [DEBUG] test episode 44: reward = -92.00, steps = 93\n",
      "22:44:32 [DEBUG] test episode 45: reward = -108.00, steps = 109\n",
      "22:44:42 [DEBUG] test episode 46: reward = -88.00, steps = 89\n",
      "22:44:52 [DEBUG] test episode 47: reward = -90.00, steps = 91\n",
      "22:45:03 [DEBUG] test episode 48: reward = -92.00, steps = 93\n",
      "22:45:14 [DEBUG] test episode 49: reward = -107.00, steps = 108\n",
      "22:45:25 [DEBUG] test episode 50: reward = -87.00, steps = 88\n",
      "22:45:36 [DEBUG] test episode 51: reward = -101.00, steps = 102\n",
      "22:45:47 [DEBUG] test episode 52: reward = -93.00, steps = 94\n",
      "22:46:00 [DEBUG] test episode 53: reward = -120.00, steps = 121\n",
      "22:46:12 [DEBUG] test episode 54: reward = -104.00, steps = 105\n",
      "22:46:26 [DEBUG] test episode 55: reward = -115.00, steps = 116\n",
      "22:46:37 [DEBUG] test episode 56: reward = -86.00, steps = 87\n",
      "22:46:49 [DEBUG] test episode 57: reward = -93.00, steps = 94\n",
      "22:47:01 [DEBUG] test episode 58: reward = -102.00, steps = 103\n",
      "22:47:10 [DEBUG] test episode 59: reward = -84.00, steps = 85\n",
      "22:47:29 [DEBUG] test episode 60: reward = -160.00, steps = 161\n",
      "22:47:44 [DEBUG] test episode 61: reward = -117.00, steps = 118\n",
      "22:47:57 [DEBUG] test episode 62: reward = -103.00, steps = 104\n",
      "22:48:10 [DEBUG] test episode 63: reward = -119.00, steps = 120\n",
      "22:48:26 [DEBUG] test episode 64: reward = -129.00, steps = 130\n",
      "22:48:40 [DEBUG] test episode 65: reward = -114.00, steps = 115\n",
      "22:48:51 [DEBUG] test episode 66: reward = -87.00, steps = 88\n",
      "22:49:03 [DEBUG] test episode 67: reward = -100.00, steps = 101\n",
      "22:49:15 [DEBUG] test episode 68: reward = -92.00, steps = 93\n",
      "22:49:30 [DEBUG] test episode 69: reward = -130.00, steps = 131\n",
      "22:49:49 [DEBUG] test episode 70: reward = -158.00, steps = 159\n",
      "22:50:01 [DEBUG] test episode 71: reward = -98.00, steps = 99\n",
      "22:50:13 [DEBUG] test episode 72: reward = -92.00, steps = 93\n",
      "22:50:26 [DEBUG] test episode 73: reward = -98.00, steps = 99\n",
      "22:50:40 [DEBUG] test episode 74: reward = -105.00, steps = 106\n",
      "22:50:51 [DEBUG] test episode 75: reward = -83.00, steps = 84\n",
      "22:51:08 [DEBUG] test episode 76: reward = -107.00, steps = 108\n",
      "22:51:24 [DEBUG] test episode 77: reward = -108.00, steps = 109\n",
      "22:51:36 [DEBUG] test episode 78: reward = -96.00, steps = 97\n",
      "22:51:49 [DEBUG] test episode 79: reward = -94.00, steps = 95\n",
      "22:52:00 [DEBUG] test episode 80: reward = -85.00, steps = 86\n",
      "22:52:13 [DEBUG] test episode 81: reward = -110.00, steps = 111\n",
      "22:52:22 [DEBUG] test episode 82: reward = -85.00, steps = 86\n",
      "22:52:39 [DEBUG] test episode 83: reward = -141.00, steps = 142\n",
      "22:52:55 [DEBUG] test episode 84: reward = -129.00, steps = 130\n",
      "22:53:05 [DEBUG] test episode 85: reward = -83.00, steps = 84\n",
      "22:53:17 [DEBUG] test episode 86: reward = -101.00, steps = 102\n",
      "22:53:30 [DEBUG] test episode 87: reward = -110.00, steps = 111\n",
      "22:53:43 [DEBUG] test episode 88: reward = -100.00, steps = 101\n",
      "22:53:57 [DEBUG] test episode 89: reward = -97.00, steps = 98\n",
      "22:54:27 [DEBUG] test episode 90: reward = -166.00, steps = 167\n",
      "22:54:46 [DEBUG] test episode 91: reward = -99.00, steps = 100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:55:22 [DEBUG] test episode 92: reward = -169.00, steps = 170\n",
      "22:55:41 [DEBUG] test episode 93: reward = -85.00, steps = 86\n",
      "22:56:03 [DEBUG] test episode 94: reward = -91.00, steps = 92\n",
      "22:56:31 [DEBUG] test episode 95: reward = -111.00, steps = 112\n",
      "22:57:00 [DEBUG] test episode 96: reward = -109.00, steps = 110\n",
      "22:57:24 [DEBUG] test episode 97: reward = -89.00, steps = 90\n",
      "22:57:45 [DEBUG] test episode 98: reward = -86.00, steps = 87\n",
      "22:58:02 [DEBUG] test episode 99: reward = -71.00, steps = 72\n",
      "22:58:02 [INFO] average episode reward = -107.07 ± 25.88\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29e3ycZZn//74mM5nJYdI2adLzEVoOLVhoqfXAYRWhoFLUXcFV8bgVvvpT1++uivp1V138qftVXHRFUVzEE+CigAsoIAdFOQVabNpCaUtPaUjSpM1MDnO+v3/M8yTTdCZzTOaZzPV+vebVyf3MPHPPNPnM9Vz3dX8uMcagKIqiVBeuck9AURRFmXpU/BVFUaoQFX9FUZQqRMVfURSlClHxVxRFqULc5Z5ArsyePdssXbq03NNQFEWpKJ599tkjxpjW8eMVI/5Lly6lvb293NNQFEWpKERkf7pxTfsoiqJUISr+iqIoVYiKv6IoShWi4q8oilKFqPgriqJUISr+iqIoVYiKv6IoShVSlPiLyN+JyHYRSYjIunHHrhWR3SLyoohcnDK+VkS2WcduEBEpZg6Koij58MgLPeztHSz3NMpOsZF/B/B24I+pgyJyOnAlsArYCHxPRGqswzcCm4EV1m1jkXNQFEXJid5gmA/95Bn+9vtPsLunur8AihJ/Y8xOY8yLaQ5tAm4zxoSNMS8Du4H1IjIPaDLGPGGSXWRuBS4vZg6Koii58sCOV0gYiMYTvPfmpzh0dLjcUyobk5XzXwAcTPn5kDW2wLo/fjwtIrJZRNpFpL23t3dSJqooSvnpPDbCd/7wErt7gpP6Ovdt62L57AZu3/waBsMx3nvz0/QGw5P6mk4lq/iLyEMi0pHmtmmip6UZMxOMp8UYc5MxZp0xZl1r6wm+RIqiVDjbDg3w8V9u4bxvPMI3H9zFu374FPuODE3Ka/UNhnlybz+XnjGP0+c3ccsHzuGVgRBX/fhpBkaik/KaTiar+BtjLjTGrE5zu3uCpx0CFqX8vBA4bI0vTDOuKEqVkEgYHtrRzRU/eIK3fvdxHnmhhw++bik/+9CricUTvOfmp3hlIFTy131wRzfxhOGSM+YCsHZJMz9471p29wT54C3PMByJlfw1ncxkpX3uAa4UEa+ILCO5sPu0MaYLCIrIBqvK5ypgoi8RRVGmCaFonF88dYALr3+MD9/azqGjI3zhzafxl2vfwOfffDqvXzGbn3xwPceGo7zn5qfoH4qU9PXv63iFJS31nD6vaXTsvJWt3HDlWWw5cJSP/PRZwrF4SV/TyRRb6vk2ETkEvAa4V0R+D2CM2Q7cAewAfgd81Bhjf6rXAD8iuQi8B7i/mDkoiuJ87t7ayeu+9jCf+802Gmrd/MeVa3j0ny/gw+cux+/zjD7uzIUz+dH71nGwf5j3/9fTBEOlScccG47wl91HuPSMeYyvLr/kjHl87e1n8qeXjvCPt28lnsiYiZ5WFOXnb4z5DfCbDMeuA65LM94OrC7mdRVFqRyi8QRf/u0O5jT5+M93n82rlzWfIMCpbFjewvfefTYf+emzfPgn7fzkg+vxeWoyPj4XHtjRTSxhuHT1vLTH33nOIgKhKP92704avX/l6+84c8I5ZiORMHQeG6GlsZb6Wme2TXHmrBRFmTY8vvsIfUMRvvr2M9iwvCWn57zxtDl8852v4pO3b+WjP3+O7793LZ6awhMV92/rYuGsOlYvaMr4mA+fu5xAKMYNf3iJ+lo379mwBL/Pjd/nps5Tk/bLwBb5l3qC7OoeZFd3kF3dQXb3DBKKJphV72HzeSdx1WuW0OB1ltw6azaKokw77t7SSZPPzQWn5Fext2nNAoKhGF+4q4N/+tXzXP/ONbhc+UfjAyNRHt99hA+8blnWaP4fL1xBYCTKLX/Zxy1/2Tc6XuMSGr1uGr3u0S+ESNywuzvIUGRsnWBOk5eVc/z8/folLGtt4A87u/n6717gh3/ay9XnL+e9G5ZSV1vcVUypUPFXFGXSGI7EeGBHN5vWzMfrzl/03rNhCYFQlG/87kX8Pjdf2bQ673TMQzu6icYNl56RPuWTiojwxbeczsWr5tITDDEYjjEYihEMxRgMxwiEogxa971u4e/WLWLFnEZOmeNnRZufGfWe48733g1LeHb/Ub790C6+et8L3PTHl7n6/OW8Z8OSrKmsRMJw8Ogw+/qGOX9l6UvdVfwVRZk0HtzRzXAkzqY1GfdyZuV/XXAyAyNRfvDYXpp8Hj698dS8nn9/RxfzZ/h41cIZOT3e5RJec1Ju6alcWLtkFj/90Kt5Zl8/1z+4i3+7dyc3/XEv11xwEu9av5jaGtcJqaOXugfZ3TPISDR5VbH9SxeXPG2k4q8oyqRx15ZO5s/wsX5pc1Hn+ezGUwmMxPjeo3s4a/Es3nT6nJyeFwxF+eOuI7z3NUuKWsAtBecsbeYX/7CBJ/f28a0Hd/Gl3+7ghj+8RCSWSJs6etf6xayc08iKOX5q3aWvylfxVxRlUugbDPPHl47w4XOXFZSrT0VE+PKmVTy7v59/vWc7rzu5Jacqmj/s7CEST3CptbHLCWxY3sLtmzfwxJ4+bnvmIM0NtayY08jKOX5WpkkdTRYq/oqiTAr3businjBcXkTKJxVPjYt/u/wM3vmDJ/jOw7v5TA7pn/u2dTG3ycdZi2aVZA6lQkR47cmzee3Js8s2B23moijKpHDXlk5OmePntHmZyyvzZf2yZv527UJ++Me9vNQ9sQncYDjGo7t62bh6btFXHtMRFX9FUUrOgb5hnjtwjE1nzS/5ua+95FQavG6+cFcHSWf49DzyQg+RWCKnKp9qRMVfURzI//fLLfzoT3sn/XV2HA5woK/0nvZ3b+0E4LJXlV78Wxq9fGbjqTz1cj+/fq4z4+Pu29ZFq9/L2iXOSvk4BRV/RXEYgVCU3z5/mK/et5Nn9vVP2uvs7R3kHTf+hbff+Be6BkZKdl5jDHdt7WT90mYWzqov2XlTufKcRZy1eCZfvW8nx4ZPNIAbjsR45MUeLlk9lxpN+aRFxV9RHMaOwwEgucD5ydu2EiiRuVkqkViCT9y2lVq3i5FIjI/89FlC0dI4Wm4/HGBP79CkpHxsXC7h3y5fzdHhCN/4/YnNBB99sZdQNMElGbx8FBV/RXEcHZ0DAHznXWfxSiDEv9y9veSv8e2HdrGtc4Cvv+NMrr9iDX89NMDnfr1twhx6rty1pRNPjfDmSc61r5o/g/e/dhm/fPoAWw4cPe7Yfdu6aGmoZf2y4vYXTGdU/BXFYWw/HGBuk4+LVs3l429YwW+2dI7m0EvBk3v7uPGxPVx5ziI2rp7LRavm8skLV/DrLZ3c/PjLRZ07njDc8/xhzl/Zxsz62hLNODOfumglbX4vX7irg1g8AST7Bjz8Qg8Xa8pnQlT8FcVhdHQOjLpPfvRvTmLtkll84TcdHOwvfmF2YDjKp27fytKWBv7PW04fHf/4G1Zw0elz+Op9O3n8pSMFn//JvX30BMNcPokpn1QavW6++JZVbD8c4KdP7geSKZ/hSHzSrzwqHRV/RXEQw5EYe3oHWTU/6UPjrnHx7SvWYIBP3VFcoxFjDJ+7axs9wTDfvmLNcV4xLpfwrSvWcHJbIx/75XMFVwDdtaWTRq+bC0/LzX6hFFx6xlzOW9nKNx/YRXcgxP0dXcyq9/BqTflMSLGdvP5ORLaLSEJE1qWMLxWRERHZat2+n3JsrYhsE5HdInKDlNtwQ1EcxM6uIAkDqxeMmZAtaq7nK5ev4pl9R7nx0d0Fn/vXz3Vy71+7+Mc3reRVi2aecLzR6+am964jkTBs/mk7Q+H8etqGonF+1/EKF6+aW3TzlXwQEb582Soi8QRfvLuDP+zs4eJVc3EX4f9fDRT76XQAbwf+mObYHmPMGut2dcr4jcBmkn19VwAbi5yDokwbth9OLvaObzpy+ZoFXPaq+Vz/0EtsPXgs7/Pu7xvii3d3sH5ZM1eff1LGxy2d3cB3//5sdnUH+ef/fj6vBeCHX+ghGI5NWconlaWzG/joBSfz++3dDIZjurErB4oSf2PMTmPMiXVWGRCReUCTMeYJk/ytuhW4vJg5KMp0oqNzgJaGWuY2+Y4bFxG+cvlq5jb5+MRtW/KKymPxBJ+8fSsul3D9FWuyLoKet7KVz15yKvdte4X/fCT3K427tnTS6vfy2pPK41fzkfOXs7Slnpn1npJaMk9XJvO6aJmIbBGRx0TkXGtsAXAo5TGHrLG0iMhmEWkXkfbe3t5JnKqiOIOOzgCrFsxIaz88o87D9Ves4WD/MF/6be7ln995eDdbDhzjq287gwUz63J6zj+cu5zL18znmw/u4g87u7M+fmA4yqMv9vLWM+eXrcLG56nhJx9cz08+sL6olo/VQtZPSEQeEpGONLdNEzytC1hsjDkL+BTwCxFpAtL9VmS8rjTG3GSMWWeMWdfaWvpONoriJMKxOLu6g6yen9kIbf2yZv7XBSdzR/sh7t/WlfWcz+7v5zsPv8Tbz17AW/OwWhARvvaOM1k1v4lP3LaV//rzy7x8ZCjj4+/r6CIST5Ql5ZPKkpaGtOsZyolktXQ2xlyY70mNMWEgbN1/VkT2ACtJRvoLUx66EDic7/kVZTqy65VBYglz3GJvOj5x4Qr+9FIvn7nzrzz6Yi9+n5tGnxu/z4Pfa99PNh3/5O1bWTCrji9dtirv+fg8Nfzgvev40C3P8KXf7uBLv93BkpZ6zl/ZyvkrW9mwvGW0YuiuLZ0sn93AGVnmrjiHSfHzF5FWoN8YExeR5SQXdvcaY/pFJCgiG4CngKuA70zGHBSl0uiwF3vnTyygnhoX/3HlWXzy9q08uquHYCjGcCS9NUONS7jjI6/B7yusQciCmXX87pPnsb9viMd29fLYi738qv0Qtz6xn9oaF+csm8X6pS08va+fT75xZdm7ZSm5U5T4i8jbSIp3K3CviGw1xlwMnAd8WURiQBy42hhjO1RdA9wC1AH3WzdFqXo6Ogfw+9wsas6el186u4G7Pvq60Z/jCcNgOEYwFD2u6fi8mT5OnVu8n/6Slgauek0DV71mKeFYnPZ9R0e/DK5/aBcisGlNeVM+Sn5IKbw8poJ169aZ9vb2ck9DUSaNTf/5Z+o9Nfxy84ZyTyUvugZG6BuMZE1XKeVBRJ41xqwbP65L4oriAKLxBDu7AifU91cC82bUqfBXICr+iuIA9vQOEoklVESVKUPFX1EcQEdn0sN/VZbFXkUpFSr+iuIAOjoHqK+tYdnshnJPRakSVPwVxQFsPzzA6fOa1H9emTJU/BWlzCQShu2HA5rvV6YUFX9FKTMv9w0xHImzagJbB0UpNSr+ilJm7J69GvkrU4mKv6KUme2HA9S6XZzc1ljuqShVhIq/opSZjs4BTpvrVxtiZUrR3zZFKSPGGDo6B1ilKR9lilHxV9Kyv2+Ih1/I3sRDKY5DR0cIhGJZnTwVpdSo+CsnkEgYPvaLLXzil1vLPZVpz9hir1b6KFPLpPj5K5XNvdu62GaJUjxhdOPRJNJxeAC3S1g5x1/uqShVhkb+ynFEYgn+7wMvjv48mEejcCV/OjoDrJjjx+epKfdUlCqjKPEXkX8XkRdE5K8i8hsRmZly7FoR2S0iL4rIxSnja0Vkm3XsBtHWP47i9mcOsL9vmEtWzwUgGIqWeUbTF3uxd6KevYoyWRQb+T8IrDbGnAnsAq4FEJHTgSuBVcBG4HsiYoc2NwKbSbZ2XGEdVxzAUDjGf/zhJV69rJm3nJnsyqSR//HEE4ZSNUDqDoTpG9ImKEp5KEr8jTEPGGNsdXiSsebsm4DbjDFhY8zLwG5gvYjMA5qMMU+Y5F/QrcDlxcxBKR0/+tPLHBmM8NlLTsXvSy4HBUOFi388YbjhDy8RmCZXD4mE4fVff5jbnjlYkvPpYq9STkqZ8/8gY/14FwCpfyGHrLEF1v3x42kRkc0i0i4i7b29vSWcqjKeI4NhbvrjHi5ZPZezFs8aFf/BIsR/Z1eAbz24i0de6CnVNMvKUCRG10CIF18JluR8HYcHEIHT5qn4K1NP1mofEXkImJvm0OeNMXdbj/k8EAN+bj8tzePNBONpMcbcBNwEyR6+2eaqFM53H95NKJbgny4+BQC/zwNQVNQeGEk+t5irBydhv4+jw5GSnK+jM8BJrY3U12rRnTL1ZP2tM8ZcONFxEXkf8BbgjWYsGXoIWJTysIXAYWt8YZpxpYzs7xvi50/t54pzFnFSa9JfphRpH/uLY7qkfcbEvzTvZ/vhAV69rLkk51KUfCm22mcj8BngMmPMcMqhe4ArRcQrIstILuw+bYzpAoIissGq8rkKuLuYOSjF880HduF2ufjkG1eMjo2mfYpY8A1YYjl9Iv+k6B8rQeR/ZDBM10BIF3uVslHs9eZ3AS/woFWx+aQx5mpjzHYRuQPYQTId9FFjTNx6zjXALUAdyTWC+084qzJldHQOcM/zh/nY35xMW5NvdLzOU0ONS4oq9QyOiv90i/yLF//th7Vnr1JeihJ/Y8zJExy7DrguzXg7sLqY11VKx9d/9wKz6j1sPn/5ceMiQqPXXdSCry360yXyt9NXx4aK/zKzK31O1xp/pUzoDt8q5vGXjvCnl47wsTesoMla4E2l0esuSriD0y7tY72fcIxILFHUuTo6B1jSUs+MuhM/d0WZClT8q5REwvD1373Agpl1vGfD4rSP8fvcBIvI+Y9F/tMr7QNwbKS41E/H4QF18lTKiop/lWKbt/3vi1bidaf3lWnyeYoS7sDI9Ir8U6uWjhVR8TMwHOVg/wirdHOXUkZU/KsQ27zt1Ll+Nq3JuMeORl+RaZ+wVeo5Ml0i/7H3cXSo8Mh/+2FrZ69G/koZUfGvQu7b1sX+vmE+vfGUCe2a/T53UaWe0zXnD8XV+ndY4r9KF3uVMqLiX4Xc/sxBFjfXc8HKtgkfV6oF38FIjESi8jdoB0MxWhpqgeLKPV/qHqTV76Wl0VuqqSlK3qj4Vxn7+4Z4Ym8f71y3EFeWJi1+n6ckpZ7GJL8AKp1gKMqi5nqgOPHvG4owp0mFXykvKv5Vxq/aD+ES+Nu1i7I+1u9zE4knCEXjWR+bjkAoRrMVKU+H1E8wFKPN78XrdhW14Ns3FKG5QcVfKS8q/lVEPGH472cPcf7KVubO8GV9fDH+PqFonEgswYKZddY5Kn/RNxiK4fd5mFVfW9SCb99gmNnWl6KilAsV/yrij7t6eSUQ4opzskf9UJy/j/2FMX9m8kvGLvusZAKhKH6fm5n1nqIWfPuHIqNXRIpSLlT8q4jbnzlIS0Mtbzh1Tk6Pb/Qmd58WErXbz5k/TSL/RMIwGI7R5HMnI/8Cc/4jkTjDkTjNjSr+SnlR8a8SjgyGeWhnN28/ewG17tz+24tp6GJH/mNpn8qO/AcjMYxJLoI3NxQu/n1DYQBma85fKTMq/lXCb57rJJYwvHNdbikfSJZ6wpg1cz6MpX2mR+Rvv5+mumTap9AF335rrUDTPkq5UfGvAowx3N5+kLMWz2TFHH/Oz7PN3grL+R+f9inkC8RJ2O/HXvA9NhwpaO9C36Al/pr2UcqMin8V8NyBY+zuGeSKPKJ+SK32yT/KtX1wWv1eamtcFZ/2sedvL/gmTGGprD4r8te0j1Juiu3k9e8i8oKI/FVEfiMiM63xpSIyIiJbrdv3U56zVkS2ichuEbnB6uilTCK/aj9IfW0Nb3nV/Lye11hEqWeqWPp97opv5Tg+8gfoLyDv32/l/DXyV8pNsZH/g8BqY8yZwC7g2pRje4wxa6zb1SnjNwKbSbZ2XAFsLHIOygQMhWP89vnDvPmMeaM5/Fzx1LjweVwFpX0CoRgi0FibFP/pFPk3F2Hx0DcYodbtoqE2vZOqokwVRYm/MeYBY4z9V/0kxzdnPwERmQc0GWOesJq93wpcXswclIm5d1sXQ5F4zrX942n0FmbrHAxFaax143IJ/iKtoZ1AYFzaBwrr5ds3FGF2Qy16wauUm1Lm/D/I8f14l4nIFhF5TETOtcYWAIdSHnPIGlMmiTueOcjy1gbWLplV0PObCozak7thk1ca0yPyT355NaWkfY4W0M6xfyiiKR/FEWTNA4jIQ8DcNIc+b4y523rM50k2av+5dawLWGyM6RORtcBdIrIKSBfuZCyZEJHNJFNELF6cvtuUkpndPYO07z/KtZecWnCkWainfzAUxW9VCzX5POw9MljQ6zuFwEgMT43gdbvGxL+gtE9YfX0UR5BV/I0xF050XETeB7wFeKOVysEYEwbC1v1nRWQPsJJkpJ+aGloIHJ7gtW8CbgJYt25dxXsCdw2M0NroxV0zNUVWv2o/SI1LePvZE2bjJiQZtRdQ7TMSo6luekX+fp8HEcHvc+OSAsV/KMJJrY2TMENFyY9iq302Ap8BLjPGDKeMt4pIjXV/OcmF3b3GmC4gKCIbrCqfq4C7i5lDpfC7jld43dce5pfPHJyS14vGE9z53CHecGobrf7CI02/11NYnX94LPL3+zwV380rGEpaOwC4XGJZPBSY9tENXooDKDYE/S7gBx4cV9J5HvBXEXke+G/gamNMv3XsGuBHwG5gD8evE0xLHn/pCB//5RYSBvYfGZqS13z4hR6ODEbyru0fT+Fpn+Nz/kOROPEKbuiSmsYCrF2++UX+tq+PNnFRnEB+tX/jMMacnGH8TuDODMfagdXFvG4l8dyBo2z+aTvLWxs4OhyhdzA8Ja97xzMHafN7ueCU1qLO4/e5C/b2SRV/SHoEzaj3TPQ0x5L6fgDL1jm/yN/29WnRyF9xALrDdxLZ2RXg/T9+mla/l1s/tJ5Fs+rpCUy++HcHQjzyYg/vWLuw6PUFv9eddxtGY8wJC75ARW/0Gi/+Mwtw9hy1dlDxVxyAiv8kse/IEO+9+Wnqa9387EOvps3vo63JS08wNOmv/d/PHiJhyMvELRN+nwdjYCiPNozhWIJo3IyKpb3wW8mLvuPTPrPqPXmLv23q1qKlnooDUPGfBLoGRnj3j54inkjwsw+vH+372tropSc4uZF/ImH4VftB1i9rZtnshqLPV0g3L3txtyllwTd5jukT+SdtnaNYBW45Yfv6tGipp+IAVPxLTN9gmPf86CkGRqL85IPrObltzEWzrclHMBQruCduLjy6q4d9fcO8+9Wl2RdRiL9P6m7Y1H8r1dkznjAEw7FxC761RGIJRvL4v+wbVF8fxTmo+JeQYCjK+//rGQ4dHeFH71vHmQtnHnfcLrnsncTo/0d/epl5M3xcesa8kpzPP2rrnHvUnrobNvUclRr526WuTcct+CbfUz7lnv1DEbzq66M4BBX/EhGKxvnQT9rZ2RXgxveczYblLSc8ps0S/8nK+28/PMBf9vTxvtcuxVOijWSFNHQJZoj8KzXnP/7LDJKRP5BXI/e+oQgt6uujOISiSj2VMf7PXR08s6+fb1+xJmOP3MmO/H/8+D7qa2t41zmls8JoSinTzJUx8bcj/8L7AjiB8V9mkBr55yH+g2FN+SiOQSP/EjAUjnHP84f5+/WL2bQms09dm98HMCmLvj2BEPc838k71y0qaS19ITn/Me/75HO97hq87spt6DL+ywxIsXXOL+2ji72KU1DxLwGPvNhDOJbgrVmapTQ31FLjkkmp9b/1if3EEoYPvG5pSc9bSL7erudvqhsTS7/PU7ELvuO/zGAs7ZPPLl877aMoTkDFvwTcv+0VZjd6OWdp84SPq3EJLQ21Jc/5j0Ti/Oyp/bzptDksaSm+vDOVhtoaRPLr4xsMxXAJxy1sNlVwN690aR/b0z+fXb59g+rrozgHFf8iGYnEefiFHjaunkONK/tCXluTt+Q5/19vOcSx4SgfPnd5Sc8LICI0evPz9wmGYjR63cctbFays2dqC0cbT40Lv9edc85/OBJjJKq+PopzUPEvkkdf7GEkGufS1bmVVrb5fSXN+ScShpsff5kzFszgnKWFNWzJRpPPk2ed//G7YYGK7uY1ft+CzcyG3Hf52tYOmvZRnIKKf5Hc1/EKzQ21rF82ccrHps1f2l2+j+7qYW/vEB8+d9mklRAmI/986vxjJwhlJUf+gVCU2hoXPs/x9fnNedg6q7WD4jRU/IsgFI3z8M5uLl41N2cDtVa/l77BcMnsjW9+/GXmNpVuU1c6/D53njn/6HE18WBfPVRm5J/uywySi765Lvja4q85f8UpqPgXwWO7ehmKxLn0jHRdLtPT5veSMGP2vsWw43CAP+/u4/2vK92mrnTk6+mf2sXLppIj/2Aodlzlkk0+5m5HBm07Z835K85Axb8I7t/Wxcx6T9rdvJlotWv9S1DuefPjL1PnKe2mrnTkm69P7eKVeo7hSJxoPFHq6U06SUfPDJF/jtU+mvZRnEaxbRy/IiJ/tbp4PSAi81OOXSsiu0XkRRG5OGV8rYhss47dIBW61z0ci/PQzh4uOn1OXlF3W1NpdvmObepaOOkNUvJP+6TP+UN+O4WdQqa0z6z6WoLhGJFY9i8029enXn19FIdQbOT/78aYM40xa4D/Ab4IICKnA1cCq4CNwPfsnr7AjcBmkn19V1jHK47HXzrCYDiWd669tbE0/j4/fdLe1LWsqPPkgt/rznmDVrKRS2bxr8TUTzAUxe898Qu2uSE5dmwke+rnyKD6+ijOoijxN8YEUn5sAOxVzE3AbcaYsDHmZZL9eteLyDygyRjzhEkaod8KXF7MHMrFvdu6aPK5ee1Js/N6Xin8fUYicX72ZHJT19ISePZnw+9zE4klCMey2xePRJO9etOlfaAyu3lNtOALcCyHip/+obDW+CuOomhjNxG5DrgKGAD+xhpeADyZ8rBD1ljUuj9+PNO5N5O8SmDx4snNa+dDJJbgwR3dXHT6XGrd+X1/+jw1zKjzFFXu+esthzg6HOVDr5/8qB/GnD0HQzG8jROnLdLthoXK7uaVFP90C765O3v2D+nuXsVZZFUuEXlIRDrS3DYBGGM+b4xZBPwc+Jj9tDSnMhOMp8UYc5MxZp0xZl1ra3GNyEvJn/ccIRiK5VXlk0qr31vwgm/qpq5c9xYUy5inf3bhHt/Fy6apQj394wnDYDhT5J+7p7+d9lEUp5A18jfGXKkVWAIAAB0eSURBVJjjuX4B3Av8C8mIPrWB7ELgsDW+MM14RXH/ti78XjevX5Ffysemze+ld7Aw8X9sVy97e4f4jyvXTFn+OB9nz0y7YSs15z+Y4f0AzBp19swt8tdKH8VJFFvtsyLlx8uAF6z79wBXiohXRJaRXNh92hjTBQRFZINV5XMVcHcxc5hqovEED+zo5sLT5+B1F1a5kdzlW9iC76+3dDK70Tupm7rGM9aGMXuEm84HJ/XnSsv5B9I0crFprs9N/G1fn2at8VccRLE5/6+JyClAAtgPXA1gjNkuIncAO4AY8FFjjL1aeA1wC1AH3G/dKoYn9vRxbDjKJasLS/lAspdvTyCMMSbv6P3lI4Osmt80qZu6xmMLXy5lmnZk3zRNIv/R91N34p9KXW2yT0G2BV/19VGcSFHib4x5xwTHrgOuSzPeDqwu5nXLyf0dXTTU1nDeysLXIFobvYRjCQKhGDPS7BydiAN9w5y1aHIM3DJhL/jmItzpGp9A0gXT53FVXM4/05WMzaz62qwLvrrBS3EiusM3D2LxBL/f3s0bT5tzgslXPhS60WtgOEogFGNxc33Br10Ioxu0cljwTdf4xCZfd1AnkKl6yWZmvSfrgq9t5aHVPoqTUPHPg6de7qd/KFJwlY9Na4GN3A/0DwOwuGVqxX9swTd71B4IRalxSdqdrJXo7xMM5xD5Z8n522mf2VrnrzgIFf88uG9bF3WeGs5f2VbUedoK3Oi1v38IYMojf6+7htoce/DaG6LSrWUkWzlWWtpn4si/uSG7+Kujp+JEVPyBR17o4fCxkQkfE08Yfr/9Fd5waht1RfqzFGruZkf+i6ZY/CG5gBvMKe2TviYekgJaaX18c0n7ZF3wVV8fxYEUvcO30tl3ZIgP3PIMbpfw5jPn8Q/nLmf1ghknPO6Zff0cGYyUpMSyyefG63blXet/sH+Yloba0QXYqSTXVo6ZfHAgmfPvzPIl6zQCI1Fq3a6MZb2zLE//RMLgytDGs28wwuxGr/r6KI6i6sXfjvgvOKWVh3Z0c/fWw7xmeQv/cN4yLljZNvoHfd+2LnweFxecUvxOYxGhrclLTyD/nP9U5/tt/D4Pgznl/CeO/Cst5x8IxU4oW01lZr2HhEleIWRyV+0fCmvKR3EcVS/+3dai67WXnsbsRi+3PX2A//rzPj54SzsntzXy4dcvY9OaBdzf8QoXrGyjoURRd2tj/u0cD/QPc/biqS3ztMk98o+xYGZd2mNNdZXXzSuYph9xKra/T/9wJKP496mvj+JAqj7nb+fd2/xeZtR5+Mj5J/Gnz/wN375iDV63i8/+ehvrv/oQvcEwlxRZ5ZNKm9+X14JvNJ7g8LHQlC/22uTq6R8YiWaMlP1eN6FooqIaugSzRP7NOVg89A2qtYPiPDTyD4Spr605Lo/uqXFx+VkL2LRmPk/s6eOHf9rLvr5h3njanJK9bluTlyf29uX8+MPHRognTFkWeyH3Vo7BUDRty0M4fpdvpUTC2SJ/29xtol6+fUNh3d2rOI6qF/+eYIg2f/rFOBHhtSfP5rUnF2bgNhFtfi8DI1FC0XhOG8ZGa/zLJP5NOZRpGpPZARNS/H1GohUk/jHmNPkyHh+zdU7/2QxHYoSiCfX1URyHpn2CYdom+OOeLPJt6mKL/5KyLfgm0z7JHjzpGYrESZjMZZGV6O8zUekqpIh/hsh/1NdH0z6Kw1DxD4RGN11NJW1WrX+u5Z4H+oeprXExxz/1X1SQXPA1Jinwmcjmg2Ongypp0Tdb2sfvc+OSCcR/SE3dFGdS1eJvjKEnGJ7wsn6yGLV4yHGj14G+YRY212WsJZ9s/Dk4e2bbEDVmDV0ZkX8snmAoEp8w8ne5xLJ4SP+F1q++PopDqWrxHwzHGI7EyxT522mf3Gr9D/QPly3fD7n5+9hdvDJG/hXWzcuubpoo8gd7l+/EaR/19VGcRlWLv11nb7tsTiUtjV5cQk61/sYYDvQNs6SM4p9L1J7Jy3/8OSol55/tSsYmaeuc/gutT319FIdSbCevr4jIX0Vkq4g8ICLzrfGlIjJijW8Vke+nPGetiGwTkd0icoOUcc97t7XDthx59BqX0NLozWnBd2AkSjAcK1uZJyRr9GFiW+dAlpy/XU5bKeZuY128Jhb/mRM4e/YPRfB51NdHcR7FRv7/bow50xizBvgf4Ispx/YYY9ZYt6tTxm8ENpNs7bgC2FjkHAqmt4yRP9jtHLOL//6+8pZ5wpigT5SyyRb5u2uSIlhpkX+6Fo6pzKr3ZBT/I4NhWhrU10dxHkWJvzEmkPJjA5C5DhAQkXlAkzHmCZOsGbwVuLyYORTD6O7eMiz4QnLRNxdP/3L5+Kcy2tAlpwXfzGKZbOhSGZF/Lu8HbFvnaNoy2H61dlAcStE5fxG5TkQOAu/m+Mh/mYhsEZHHRORca2wBcCjlMYessbLQHQjh87hGUxpTTZs/t7TPqJXzLCcs+E4k/lHcLsHnyfxrVUnmbhN1JUtlZn0tkViCkeiJZbD9Q2rtoDiTrOIvIg+JSEea2yYAY8znjTGLgJ8DH7Oe1gUsNsacBXwK+IWINAHprn0zXi2IyGYRaReR9t7e3nzfW1Z6gmHa/L6yXZK3+X0cGYwQT0x4wcTB/mFmN3pLZipXCI21lvhnyflnauRiU1nin+uCb/LKIF25Z9+gRv6KM8mqJsaYC3M81y+Ae4F/McaEgbD1/GdFZA+wkmSkvzDlOQuBwxO89k3ATQDr1q2bWCELoDsQYk6Z8v2QTPvEE4b+ocho3X86kmWe6Z0ypwqXSyxnz4lz/pl8fWz8vsxlkU4jW+mqzcxRi4fICY6m6uujOJViq31WpPx4GfCCNd4qIjXW/eUkF3b3GmO6gKCIbLCqfK4C7i5mDsXQa0X+5aItx16++/vKW+Nvky1qz2aFYJ+jUjZ5BcMxvG4Xte6J/0zGIv/jv9RsX58WrfFXHEixeYSvicgpQALYD9hVPecBXxaRGBAHrjbG9FvHrgFuAeqA+61bWegJhjn/lPL9YdpVRhPl/SOxBF0DIyxuLtvSyCh+nzvLgm/mLl5j56ikBd+JrR1sxmydj39f9gYvTfsoTqQo8TfGvCPD+J3AnRmOtQOri3ndUjAUjjEYjpU58rd6+U4g/oePjZAw5enbO55Gr5tgeOK0T7YrlKa6yon8s3XxsrHTPuPTWerroziZqt3hawtuuXP+MHHkP+bm2TAlc5qIZCvHbGmfiSPlJp+HSCxBOJbZIM4pBEMx/FnWMGDM03/8Ll/b10fTPooTqVrxt3f3ljPy93lq8PvcE/byLbePfyrZGroERqI55fyhMiwegqHMXclS8dQky4XH5/yPDGrkrziXqhX/cvr6pNLm905o63ygf5hat6ss5nPjafK5M5Z6JhKGwUgu1T6VJP7ZF7BtZjacuMu3X319FAdTveJfRl+fVNr8vgltnQ/0DbNoVvmsnFOZqNRzMBLDmOw+OPaCsF1G6WRyWcC2aU5j69w3GFZfH8WxVK/4B8PUul001ZW3k2VrFn+fA/3Djsj3QzLnn6kBe64boqZt5F9fm3bBV319FKdSveJvbfAq9x9mm+Xvk84XxhjDwTL7+Kcykb9Pti5eNpXSzSsaTzAciedU6gnpzd3U2kFxMlUr/t2B8m7wsmlr8hKKJtJaJR8bLr+Vcyq2JXO6qH26Rf6DOb4fm5n1tRwbOrHOX/P9ilOpWvHvCZbX2sFmtJ1jmtTPfgdV+kCKrXOaWv9crRDs40739M/1y8xmVn0twXCMSGwsJaaOnoqTqV7xd0rkb2/0SrPo66QyT5g4as/m5W8z0dWDk8jWmGY8zQ3Jxx0bSaZ+jDH0DYW1faPiWKpS/EcicYLh2IRmalPFaC/fNOWeBytK/HMTyxrLIK5SIv9cCwLGdvkm39dwJE4omtDIX3EsVSn+tpHanDI1cUllLPI/caPXgb5hWv1e6hxSKtg42soxTdonjzRJJdg6B0dbOOa64Dvm7Ala4684n6oU/267g5cDIv+mOje1bldai4cDDqr0gdRWjunTPrU1Lnye7F9UldDNK9+c/8xxnv62r89srfZRHEpVir+TIn8RobUxfa2/88R/4rRPrkJZSZF/zqWeDcebu/VZabzmhvIHGIqSjqoUfydF/pAs9xwf+UdiCQ4PjDhK/L1uF54aSSvcgTw2RFWC+OeTxoLkDl+Aflv81dFTcThVKf49wRC1Na7RS/Vy05amkXvnsRGMcc5iLySvUvw+T9qcf67e91AZnv7BUBSfx4WnJrc/kbraGrxu1+iCr+3lr5u8FKdSleLfGwjT6i//7l6bdBYPo2WeLc4Rf7D9fdLn/HOtjKmEyD8Xe+rxzKqvTVnwtX19ymsfoiiZKIn4i8g/iYgRkdkpY9eKyG4ReVFELk4ZXysi26xjN0gZFLg7GCq7m2cqbX4fx4ajx3ncO63G3yaTcOdjgub3eQiEomktLZxCPr4+NjPrPcct+LZovl9xMEWLv4gsAt4EHEgZOx24ElgFbAS+Z/f0BW4ENpPs67vCOj6l9ATCZXfzTMVee7D93wEO9A3hdbtoddgmoUytHPMRy6Y6N9G4IRw70SDOKQTySGPZzEoxd+sbVF8fxdmUIvK/Hvg0kBrGbQJuM8aEjTEvA7uB9SIyD2gyxjxhkmHfrcDlJZhDXnQHnBX5j1o8pNT625U+TrByTqXR60m7QSufNEklWDwEc2zhmEpzQ+3ogq9aOyhOpyjxF5HLgE5jzPPjDi0ADqb8fMgaW2DdHz+e6fybRaRdRNp7e3uLmeoooWicQCjmiDJPm3S9fA/0O6vSx6bJ5z7BhC6eMAyG84j8K8DcLdnFK7/If2a9Z3TBt1/TPorDyfrXKiIPAXPTHPo88DngonRPSzNmJhhPizHmJuAmgHXr1pUkQWx76DjB2sHGvgqxyz1tK+dXL2su57TSkq6VY74OmJXg7FlIzt9O+yQShiODYU37KI4m62+3MebCdOMicgawDHjeWrNdCDwnIutJRvSLUh6+EDhsjS9MMz5l2CWVTqnxh2QtuMhY5H90OMpgOObIyN9vRf7GmNFqKTt9k62F49g5nO/pX+iCb8IkCwrCMfX1UZxNwWkfY8w2Y0ybMWapMWYpSWE/2xjzCnAPcKWIeEVkGcmF3aeNMV1AUEQ2WFU+VwF3F/82cscWWCelfdw1Lloaaum1vpj29w0Bzqv0gWTOP54wjETHKpNydfS0sUU1MOLMyD8aTzASzb2Ri43t77O7ZxDQDV6Ks5mUImRjzHYRuQPYAcSAjxpjbLW4BrgFqAPut25TRnfAeZE/QGtKL1+7zHOJw2r84fiUjV3Dnq8VQpPDI/98fX1s7Eh/VPw17aM4mJKJvxX9p/58HXBdmse1A6tL9br50hMM46mR0SjNKbT5vaO2zraV88JZzhb/OU2M3k89ls85nEi+X2Y29o5xW/zV10dxMlW3w7c7EKK10eu4EspWv/e4yL/NQVbOqYwJ91jUnm/jk4ZaNyLTL/K3A4o9vZr2UZxP1Yl/bzBMm4Py/TZtfi9HBsMkEsZxbp6p2AKfWu6Zr1i6Rhu6ODPyH/syK1T8k2s2mvZRnEzViX+yfaPzLsfb/F5iCUP/cIQDfcOO8/SxSdeGMViAWCY9/Z0p/mML2Pmlffw+Ny5JBhjq66M4naoTf6f5+tjYVyOdR0foCoQcHPmfmPYJhmJ43S687tzTVH6fc1s5Fir+LtfYWpJu8FKcTlWJfzgW59hw1FG+Pjb2prMtB446zso5Fdu8LTVqDxTggOnkbl6FXMnY2Iu+mvJRnE5Vib+9oOrIyN8S//b9RwHnin9jmkqdpBVCfkLpZFtne/9BYwHiPxb5q/grzqa6xD9oi7/zIn/b3+dZh4t/jUtoqK05bsE3ny5eNk4W/2AoSp2nJudGLqnMtMRfyzwVp1Nd4u/QDV6Q7ATl97rpGgjh87gc5T00nqS/T2rOP3/7Yyd38yrE2sFmlqZ9lAqhusTfjvwdmPOHsbz/4uZ6x3QZS0eylePxpZ65dvEaO0cy8ndiQ5dgOPdm9OOxd/lq2kdxOlUm/iFqXOLYP8xU8Xcy41s55tPFy8bv8xAb5xHkFApp4WgzlvZx5u+YothUlfh3B8KO3N1rY69FLHK4+CfLNI+P/PONlO0rBSfm/QtZw7DRtI9SKVSV+PcEw8xxYKWPjd2y0emRf5PPw6CVr4/FEwxH8nfAdLKtczAUzdmeejxzZyS/wOfNqCvllBSl5FSX+AdCtDo03w9jJahOdPNMJTXtU6gPzqitswMj/0JaONqct6KVO695DafNayrxrBSltFSX+Ds88l9iRfwntTaWeSYTYzd0gcLF38mtHAMj+Vcv2bhcwtolzuvApijjqRrxj8QS9A9FHFvpA3DRqrnc9/FzWdLSUO6pTEijz81wJE4snsi7i5eNU9M+kViCcCyB36u+PMr0piTiLyL/JCJGRGZbPy8VkRER2Wrdvp/y2LUisk1EdovIDTJFNY22V74Td/fa1LiE0+c7P12Q6uxZeOSfPIfTunkVY+2gKJVE0b/hIrIIeBNwYNyhPcaYNWmeciOwGXgSuA/YyBR087I3eDk57VMp+FOcPW2xLMQBM3kOZ0X+Y19mhaV9FKVSKEXkfz3waSDrbh0RmQc0GWOeMMndPbcCl5dgDlnpDjh7g1clkdqJq9DIv762hhqXOC7nX+j7UZRKoyjxF5HLgE5jzPNpDi8TkS0i8piInGuNLSDZ6N3mkDWW6fybRaRdRNp7e3uLmepoc3Qnp30qhdS0T75dvGxExKoaclrkX9j7UZRKI2t4IyIPAXPTHPo88DngojTHuoDFxpg+EVkL3CUiq4B0+f2MVwzGmJuAmwDWrVtXlA9AdyCMS9RnvRQ0pqRsiomUnWjuFtDIX6kSsv6GG2MuTDcuImcAy4DnrTXbhcBzIrLeGPMKELae/6yI7AFWkoz0F6acZiFwuKh3kCM9wRCzG73UOHR3byVhC2NywbdwB0y/z+O4On878p9R4CYvRakUCk77GGO2GWPajDFLjTFLSQr72caYV0SkVURqAERkObAC2GuM6QKCIrLBqvK5Cri7+LeRnWSNv+b7S4G94Buwcv6FRslNDuzmpTl/pVqYrN/w84Avi0gMiANXG2P6rWPXALcAdSSrfCa90geSaZ/5M1T8S0FqjX4x4u/3eeg8NlLKqRWN/WXUqHX+yjSnZL/hVvRv378TuDPD49qB1aV63VzpDYZYs2jmVL/stMTncVHjEgZDyQXfQhdHm3xuXnBg5F9fW4O7gDSWolQSVfEbHo0nODIYcWQTl0pEREYXa4txwHTigm+yMY1G/cr0pyrE/4i1u1dz/qXD9vdJ9u8tLPK3m8I4qaFLMV7+ilJJVIX4jzZu18i/ZDR6PSXI+buJJwzDEec0dCnm/ShKJVEV4t8d0A1epcZu6FKM9739PCdV/BTSj1hRKpGqEH+7d6+mfUqH3+vm6FCEULRwB0y/A22di/HyV5RKojrEPxBCRJtqlxK/z03XQGj0fmHncJ6tc0Bz/kqVUB3iHwwzu9Gr5XslxF6ste8Xdg7ndfMKhKIa+StVQVWoYXcgpIu9JaYxRSCL2eELzkn7hGNxIrGELvgqVUFViL9aO5Qe/3HiX3ipJzgn7aNe/ko1UTXir5F/aUld5G2qKzTyt8XfGZG/+voo1cS0F/9YPMGRQRX/UpMaHRe6ycvnceF2CYERp0T+6uWvVA/TXvz7hiIYA22a9ikpqcZnhUbKqTYRTkAjf6WamPbiP7rBSyP/kpIqkMU4YPp9Hgfl/LV5u1I9THvxt60ddMG3tNipkWIdMJ0U+dslp4WmsRSlkpj+4m/t7lVrh9JiR8fFRslOEv+gir9SRUx78e+2dvfOblTxLyW26BcrlE0+j2O8feyF50ZN+yhVQFHiLyL/KiKdIrLVul2acuxaEdktIi+KyMUp42tFZJt17AarneOk0RMM09JQW1CPWSUzDd5SRf4eR0X+DbU12udZqQpKoYjXG2PWWLf7AETkdOBKYBWwEfie3dMXuBHYTLKv7wrr+KTREwjR6td8f6nx1Lio89QUXRbpd1AfX3X0VKqJybq+3QTcZowJAy+LyG5gvYjsA5qMMU8AiMitwOVMYh/f5O5eTflMBo0+d9GRf5OV83/Ttx4r0awK55WBEHO1z7NSJZRC/D8mIlcB7cD/NsYcBRYAT6Y85pA1FrXujx9Pi4hsJnmVwOLFiwua3DlLm5k/U/+gJ4N/vugUFrfUF3WOS8+cx94jQyQc0M1rxZxGLjilrdzTUJQpQbK10BORh4C5aQ59nqTAHwEM8BVgnjHmgyLyn8ATxpifWee4GbgPOAD8/8aYC63xc4FPG2Pemm2i69atM+3t7Tm/MUVRFAVE5FljzLrx41kjf1uoc3iBHwL/Y/14CFiUcnghcNgaX5hmXFEURZlCiq32mZfy49uADuv+PcCVIuIVkWUkF3afNsZ0AUER2WBV+VwF3F3MHBRFUZT8KTbn/w0RWUMy7bMP+AiAMWa7iNwB7ABiwEeNMXaX7muAW4A6kgu9k7bYqyiKoqQna87fKWjOX1EUJX8y5fx155OiKEoVouKvKIpShaj4K4qiVCEq/oqiKFVIxSz4ikgvsL/Ap88muRmtEqikuUJlzbeS5gqVNd9KmitU1nyLnesSY0zr+MGKEf9iEJH2dKvdTqSS5gqVNd9KmitU1nwraa5QWfOdrLlq2kdRFKUKUfFXFEWpQqpF/G8q9wTyoJLmCpU130qaK1TWfCtprlBZ852UuVZFzl9RFEU5nmqJ/BVFUZQUVPwVRVGqkGkt/iKy0Wogv1tEPlvu+WRDRPZZze23iojjXOxE5Mci0iMiHSljzSLyoIi8ZP07q5xztMkw138VkU7r890qIpeWc442IrJIRB4RkZ0isl1EPmGNO/WzzTRfx32+IuITkadF5Hlrrl+yxp362Waab8k/22mb87caxu8C3kSyicwzwLuMMTvKOrEJsHocrzPGOHLziYicBwwCtxpjVltj3wD6jTFfs75gZxljPlPOeVrzSjfXfwUGjTH/t5xzG4/VF2OeMeY5EfEDz5Lsbf1+nPnZZprvO3HY52v1DWkwxgyKiAd4HPgE8Hac+dlmmu9GSvzZTufIfz2w2xiz1xgTAW4j2VheKRBjzB+B/nHDm4CfWPd/QlIEyk6GuToSY0yXMeY5634Q2Emyt7VTP9tM83UcJsmg9aPHuhmc+9lmmm/Jmc7ivwA4mPLzhM3iHYIBHhCRZ63m9ZXAHKtDG9a/Tu+A/jER+auVFnLEpX4qIrIUOAt4igr4bMfNFxz4+YpIjYhsBXqAB40xjv5sM8wXSvzZTmfxlzRjTs9xvc4YczZwCfBRK3WhlI4bgZOANUAX8M3yTud4RKQRuBP4pDEmUO75ZCPNfB35+Rpj4saYNSR7hq8XkdXlntNEZJhvyT/b6Sz+mZrIOxZjzGHr3x7gNyRTV06n2+7lbP3bU+b5ZMQY0239YSWAH+Kgz9fK794J/NwY82tr2LGfbbr5OvnzBTDGHAMeJZk/d+xna5M638n4bKez+D8DrBCRZSJSC1xJsrG8IxGRBmvxDBFpAC4COiZ+liO4B3ifdf99wN1lnMuE2H/sFm/DIZ+vtch3M7DTGPOtlEOO/GwzzdeJn6+ItIrITOt+HXAh8ALO/WzTzncyPttpW+0DYJVDfRuoAX5sjLmuzFPKiIgsJxntA7iBXzhtviLyS+ACkhaz3cC/AHcBdwCLgQPA3xljyr7QmmGuF5C8bDbAPuAjdt63nIjI64E/AduAhDX8OZJ5dCd+tpnm+y4c9vmKyJkkF3RrSAa7dxhjviwiLTjzs800359S4s92Wou/oiiKkp7pnPZRFEVRMqDiryiKUoWo+CuKolQhKv6KoihViIq/oihKFaLiryiKUoWo+CuKolQh/w+c/SLekWferAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -120:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
